"""**********************************
   Author: yuhengShi
   Time: 2022/1/24 11:08
   Project: pytorch_classification
**********************************"""
import string

import jieba
from pandas import DataFrame

from config import conf


class Preprocess:
    def __init__(self):
        pass

    @staticmethod
    def tokenize(sentence):
        """
        分词 根据conf.tokenize_level级别
        :param sentence:
        :return:
        """
        if conf.tokenize_level == "char":
            words = [x for x in sentence]
        else:
            words = jieba.cut(sentence)
        return words

    @staticmethod
    def split_data(data_df: DataFrame, train_ratio=0.8):
        df = data_df.sample(frac=1)
        df_len = len(df)

        num = train_ratio*df_len
        train = df[:num]
        test = df[num:]
        return train, test

    @staticmethod
    def del_blank(text: str):
        text = text.replace(" ", '')
        return text
        pass

    @staticmethod
    def remove_punctuation(content: str, remove_blank=True, lower=True):
        """
        把英文文本里的符号去除 以及空格
        :param content:
        :param remove_blank: default True ,去除空格
        :param lower: default True ,小写
        :return:
        """
        # 将空格也去除
        punctuation = string.punctuation + ' ' if remove_blank else string.punctuation
        content = content.translate(str.maketrans("", "", punctuation))
        content = content.lower() if lower else content
        return content

    @staticmethod
    def padding(seqs: list, padding_index: int, padding_len: int):
        """
        padding到指定长度或者截断
        :param seqs:
        :param padding_index:
        :param padding_len:
        :return:
        """
        seqs_len = len(seqs)
        if seqs_len < padding_len:
            diff = padding_len - seqs_len
            seqs = seqs + [padding_index] * diff
        else:
            seqs = seqs[:padding_len]
        return seqs
